#!/usr/bin/env python
#
# Greatly simplified representation of a Condor queue and associated
# scheduling.
#
# Author:   Mike Murphy (mamurph@cs.clemson.edu)
# Revision: 23 October 2009
#
#   Copyright 2009 Clemson University
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
# This material is based upon work supported under a National Science
# Foundation Graduate Research Fellowship. Any opinions, findings,
# conclusions or recommendations expressed in this publication are those of
# the author(s) and do not necessarily reflect the views of the National
# Science Foundation.

'''
Simulation of the Condor scheduler (negotiation process only). This component
uses the extra_data fields of both machines and jobs to store Condor classAds
for matching purposes. The extra_data dictionaries must contain a classAd
key, which nominally references another dictionary containing name: value
pairs. However, any type for which the == operator compares equal should work.
It should be noted that only exact match classAds are supported: matches
using inequalities, functions, or other techniques are not supported. Strings
used in classAds are case-sensitive.

If extra_data is None, or the classAd is missing, for any job, then the default
behavior will be to match the job to ANY machine. Similarly, if a machine has
no extra_data or the classAd is missing, then the machine may be matched to
ANY job. This behavior can be changed by setting the require_classAds
property of a Condor instance to True, which will cause an Exception to be
raised whenever classAds are missing.
'''


import job_model, scheduler

DEFAULT_NEGOTIATOR_INTERVAL = 300


class Condor(scheduler.Scheduler):
   '''
   Simulation of the Condor interval scheduler (negotiation process only).
   Scheduling is performed only during negotiation intervals, utilizing a
   periodic process. The scheduling discipline is not completely FIFO/FCFS,
   since the negotiation process must match job and machine classAds. As a
   result, long delays are possible for some jobs, while others are matched
   more quickly. Moreover, if a job is never matched to a machine, it will
   remain in the queue forever.

   In order to increase simulation speed by reducing the number of events
   that must be processed, this scheduler automatically suspends itself
   when no jobs are waiting in the queue. When new jobs are submitted, the
   interval scheduler resumes scheduling as long as jobs are waiting, using
   an offset algorithm to keep the actual scheduling runs consistent with
   the interval setting.
   '''
   def __init__(self, simobj, tag='', interval=DEFAULT_NEGOTIATOR_INTERVAL, \
                **kwds):
      '''
      @param simobj: Reference to the simulator object
      @param tag: Optional scheduler tag
      @type tag: str
      @param interval: Scheduler interval
      @type interval: number
      @param kwds: Extra data to be passed to the Scheduler constructor
      '''
      scheduler.Scheduler.__init__(self, simobj, name='Condor', tag=tag, **kwds)
      self.interval = interval
      self.require_classads = False
      self.start_time = simobj.get_time()
      self.last_sched_run = self.start_time
      self.check_schedule()
   #
   def check_schedule(self):
      '''
      Determines if a scheduler event is needed and adds the event if
      necessary.

      @rtype: bool
      @return: True iff a new scheduler event has been added
      '''
      result = False
      if not self.event_pending():
         if self.get_waiting_count() > 0:
	    cur_time = self.sim.get_time()
	    adjusted = self.last_sched_run - self.start_time
	    offset = self.interval - (adjusted % self.interval)
	    when = cur_time + offset
	    self.add_scheduler_event(when)
	    result = True
      return result
   #
   def pick_slot(self, job):
      '''
      Selects a free machine, if available, on which to run a job. Matching
      of job and machine classAds is performed in this method.

      @param job: Job to be scheduled for execution

      @return: a free machine which matches the job, or None if no machine is
               free or if the classAds cannot be matched
      '''
      slot = None
      job_ads = None
      if type(job.extra_data) is dict:
         if job.extra_data.has_key('classAds'):
	    job_ads = job.extra_data['classAds']
      #
      for m in self.free_targets:
         if not m.claimed:
	    machine_ads = None
	    if type(m.extra_data) is dict:
	       if m.extra_data.has_key('classAds'):
	          machine_ads = m.extra_data['classAds']
	       #
	    #self.debug('machine ads: %s  job ads: %s' % \
	    #           (str(machine_ads), str(job_ads)))
	    if (job_ads == None) or (machine_ads == None) or \
	                               (job_ads == machine_ads):
	       if self.require_classads and \
		         ((job_ads == None) or (machine_ads == None)):
		     raise Exception('Required classads missing')
	       else:
	          slot = m
		  break
	 else:
	    self.debug('machine %s is claimed' % m.name)
      self.debug('picked slot: %s' % str(slot))
      return slot
   def schedule(self, time, simobj):
      '''
      Periodic scheduling method: iterates through the queue, attempting to
      match waiting jobs to free slots (using pick_slot()). Starts jobs on
      machines when matches are made.

      @param time: Current simulated time
      @type time: number
      @param simobj: Reference to the simulator object

      @return: collection of simulator results (1-tuple containing a list)
      '''
      idx = len(self.queue) - 1
      r = []
      while (self.get_free_count() > 0) and (idx >= 0):
         job = self.queue[idx]
	 run_time = job.length
	 slot = self.pick_slot(job)
	 if slot != None and self.claim_slot(slot):
	    self.debug('starting job on slot %s' % str(slot))
	    s = self.start_job(job, slot)
	    if s is not None:
	       r.append(s)
	 #
         idx -= 1
      #
      # Check to determine if we need another scheduler event
      self.last_sched_run = time
      self.check_schedule()
      #
      r.append(self.get_update())
      return (r,)
   #
   def submit_job(self, job):
      '''
      Receives a job into the scheduling queue and ensures that the periodic
      scheduler is running.

      @param job: Job to be received

      @return: results collection for simulator output
      '''
      r = []
      r.append(self.enqueue_job(job))
      self.check_schedule()
      r.append(self.get_update())
      return (r,)
   #
#


def str_to_ads(raw):
   '''
   Generates a classAd dictionary from a string of the form::

              ad1=value1;ad2=value2;ad3=value3

   @param raw: Raw string in the above format
   @type raw: str

   @rtype: dict
   @return: dictionary containing classAds
   '''
   ads = {}
   lines = raw.split(';')
   for line in lines:
      stripped = line.strip()
      lhs, rhs = stripped.split('=')
      ads[lhs] = rhs
   return ads
#


def str_to_extra(raw):
   '''
   Converts a raw classAds string (see str_to_ads, above) to an extra_data
   dictionary, containing the classAds converted to a dictionary.

   @param raw: Raw string to be converted
   @type raw: str

   @rtype: dict
   @return: dictionary suitable for use as extra data (e.g. with the **
            syntax to match **kwds on Job/Machine constructors)
   '''
   return { 'classAds' : str_to_ads(raw) }
#


def ads_to_str(ads):
   '''
   Converts classAds in the dictionary form produced by str_to_ads back to
   a raw classAd string.

   @param ads: ClassAds in dictionary form
   @type ads: dict

   @rtype: str
   @return: string containing classAds in semi-colon delimited name=value
            form
   '''
   return ';'.join([ '='.join([key, ads[key]]) for key in ads ])
#


if __name__ == '__main__':
   from cluster_sim import Cluster
   from job_model import Job
   from kernel import Simulator

   sim = Simulator()
   c = Cluster('test', sim)
   c.grow(10)
   s = Condor(sim)
   c.add_scheduler('Scheduler', s)
   s.add_targets(c.machines)

   for x in xrange(0,20):
      job = Job(20 + (x % 4), name='job%d' % x, prob_err=0.2)
      s.add_submit_event(5, job)
   #

   sim.run()
#